# -*- encoding: utf-8 -*-
import os, sys
sys.path.append(os.path.join(os.path.dirname(__file__), '../'))
import pycommonlib as pyclib
import workerpool
import traceback
import datetime, time
import re
import html2textlib
import threading

from pymongo    import Connection
from termcolor  import cprint
from lxml       import etree


LOCAL_PATH = '/home/hoangnamhai/HarvestedData/horoscope'
#MONGO_SERVER = '27.0.12.106'   
#MONGO_SERVER = 'beta.mana.vn'   
MONGO_SERVER = 'localhost'   
MONGO_PORT = 27017
DATABASE = 'tuvi'
PREFIX = '/uploads/horoscope' 
SITE_URL = 'http://www.hoahoctro.vn' 
BASE_URL = 'http://www.hoahoctro.vn/horoquiz.hht'
MAX_COUNT = 15
MAX_ARTICLE = 30
MAX_PAGE = 20
os.umask(0000)

logger = pyclib.getMongoLog(MONGO_SERVER, MONGO_PORT, 'hoahoctro.vn')
totalNewsCrawlered = 0; totalNewsDuplicated = 0
flgCopy = pyclib.getArgs()
ssh = None; sftp = None
if flgCopy!=None:
    ssh = pyclib.createSSHClient('mana.vn', 22, 'daudm', '')
    sftp = ssh.open_sftp()
    if ssh==None:
        if totalNewsCrawlered == 0 and totalNewsDuplicated == 0:
            logger.critical(unicode("crawler tin tức hoahoctro.vn không hoạt động", 'utf8'))
            pyclib.forceQuit()
start = 0
    
class Hoahoctro():
    def __init__(self, server, port, database):
        CONNECT = Connection(server, port)
        self.DB = CONNECT[database]

    CUNG = {
        'maket': {'name': unicode('Ma kết', 'utf-8'), 'date': '22/12-19/01', 'img': 'http://hoahoctro.com.vn/upload/000_horoscope/2011/icon/mk.jpg'},
        'baobinh': {'name': unicode('Bảo bình', 'utf-8'), 'date': '20/01-18/02', 'img': 'http://hoahoctro.com.vn/upload/000_horoscope/2011/icon/bb.jpg'},
        'songngu': {'name': unicode('Song ngư', 'utf-8'), 'date': '20/01-18/02', 'img': 'http://hoahoctro.com.vn/upload/000_horoscope/2011/icon/sn.jpg'},
        'bachduong': {'name': unicode('Bạch dương', 'utf-8'), 'date': '21/03-20/04', 'img': 'http://hoahoctro.com.vn/upload/000_horoscope/2011/icon/bd.jpg'},
        'kimnguu': {'name': unicode('Kim ngưu', 'utf-8'), 'date': '21/04-20/05', 'img': 'http://hoahoctro.com.vn/upload/000_horoscope/2011/icon/kn.jpg'},
        'songtu': {'name': unicode('Song tử', 'utf-8'), 'date': '21/05-21/06', 'img': 'http://hoahoctro.com.vn/upload/000_horoscope/2011/icon/stu.jpg'},
        'cugiai': {'name': unicode('Cự giải', 'utf-8'), 'date': '22/06-22/07', 'img': 'http://hoahoctro.com.vn/upload/000_horoscope/2011/icon/cg.jpg'},
        'sutu': {'name': unicode('Sư tử', 'utf-8'), 'date': '23/07-23/08', 'img': 'http://hoahoctro.com.vn/upload/000_horoscope/2011/icon/st.jpg'},
        'xunu': {'name': unicode('Xử nữ', 'utf-8'), 'date': '23/08-22/09', 'img': 'http://hoahoctro.com.vn/upload/000_horoscope/2011/icon/xn.jpg'},
        'thienbinh': {'name': unicode('Thiên bình', 'utf-8'), 'date': '23/09-22/10', 'img': 'http://hoahoctro.com.vn/upload/000_horoscope/2011/icon/tb.jpg'},
        'thannong': {'name': unicode('Thần nông', 'utf-8'), 'date': '23/10-21/11', 'img': 'http://hoahoctro.com.vn/upload/000_horoscope/2011/icon/tn.jpg'},
        'nhanma': {'name': unicode('Nhân mã', 'utf-8'), 'date': '22/11-21/12', 'img': 'http://hoahoctro.com.vn/upload/000_horoscope/2011/icon/nm.jpg'},
    }    
    
    def convertText(self, text):
        try:
            text = pyclib.toAscii(text)
            text = re.sub(r'\s+', '', text)
            return text
        except:
            traceback.print_exc()

    def getCung(self, name):
        try:
            collection = self.DB['cung']
            if name==None or name=='': return None, None 
            root = collection.find_one({'data': name})
            if root==None:
                name = self.convertText(name)
                result = collection.save(self.CUNG[name])
            return result
        except:
            traceback.print_exc()
    
    def getDatetime(self, text):
       try:
           result = None
           if text==None or text=='': return result
           m = pyclib.regexString('(\d+)/(\d+)/(\d+)', text)
           if m:
               year = int(float(m.group(3))); month = int(float(m.group(2))); day = int(float(m.group(1)))
               return datetime.datetime(year, month, day, 0, 0, 0) + datetime.timedelta(seconds=time.timezone)
           else:
               m = pyclib.regexString('(\d+)/(\d+)', text)
               if m:
                   now = datetime.datetime.now()
                   year = int(float(now.year)); month = int(float(m.group(2))); day = int(float(m.group(1)))
                   return datetime.datetime(year, month, day, 0, 0, 0) + datetime.timedelta(seconds=time.timezone)
           return result
       except:
           traceback.print_exc()
   

    def checkArticleDuplicate(self, link):
        ''' Kiểm tra trùng tin tức trong DB, trả vê 1 nếu trùng, 0 trong trường hợp ngược lại
        ''' 
        try:
            collection = self.DB['article']
            if link==None or link=='': return None, None
            m = pyclib.regexString('/([a-z0-9]{1,3})/(\d+)/', link)
            if m: hashUrl = '{0}-{1}'.format(m.group(1), m.group(2)) 
            else: hashUrl = pyclib.getMd5(link)   
            result = collection.find_one({'hashUrl' : hashUrl, 'source': 'hoahoctro.vn'})
            if result!=None:  cprint('Tin tức đã tồn tại trong cơ sở dữ liệu', 'red'); return 1, hashUrl
            return 0, hashUrl 
        except:
            traceback.print_exc()
            return None, None
    
    def processArticle(self, link, title):
        ''' Hàm xử lý chi tiết một tin tức, nếu tin đã tồn tại trong DB thì trả về: 1, ngược lại: 0
        '''
        try:
            global totalNewsCrawlered, totalNewsDuplititleed
            date_time = self.getDatetime(title)
            if date_time==None: 
                print 'Không get được datetime'; pyclib.forceQuit()
        except:
            traceback.print_exc()        
            
    def processAll(self):
        try:
            gbcount = 0; loop = 0 
            for page in range(0, MAX_PAGE):
                lurl = '{0}/p{1}'.format(BASE_URL, page)
                cprint('Process page : ' + lurl, 'yellow')
                tree        =   pyclib.getXMLTree(lurl)
                firstNode   =   tree.xpath('//div[@class="newsdetail"]/div[@class="firstArticle"]/a')
                if len(firstNode) > 0:
                    node = firstNode[0]
                    title = pyclib.getStringWithNode(node)
                    link = node.get('href')
                    count = self.processArticle('{0}{1}'.format(SITE_URL, link), title)
                    if count!=None: gbcount += count; loop += 1
                listNode    =   tree.xpath('//div[@class="newsdetail"]/table[@class="newslist"]//tr/td[2]/a')
                if len(listNode) < 1: return None, None
                for node in listNode:  
                    title = pyclib.getStringWithNode(node)
                    link = node.get('href')
                    count = self.processArticle('{0}{1}'.format(SITE_URL, link), title)
                    if count!=None: gbcount += count; loop += 1
        except:
            traceback.print_exc()

def timeOut():        
    global totalNewsCrawlered, totalNewsDuplicated
    while True:
        delta = time.time() - start
        if delta > 900:
            print 'Dừng chương trình vì vượt quá thời gian chạy.', datetime.datetime.now()
            if totalNewsCrawlered == 0 and totalNewsDuplicated == 0:
                logger.critical(unicode("crawler tin tức hoahoctro.vn không hoạt động", 'utf8'))
            else:
                logger.info(unicode("Tổng số bài viết crawled: {0}, tổng số bài trùng lặp {1}".format(totalNewsCrawlered, totalNewsDuplicated), 'utf8'))
            logger.info('process timeout {0}'.format(delta))
            pid = os.getpid(); os._exit(1); os.kill(pid, 9)
        time.sleep(30)

if __name__ == '__main__':
    try:
        print 'Start crawler hoahoctro.vn: ', datetime.datetime.now()
        totalNewsCrawlered = 0; totalNewsDuplicated = 0
        start = time.time() 
        timeout = threading.Thread(target=timeOut).start()
        crawler = Hoahoctro(MONGO_SERVER, MONGO_PORT, DATABASE)
        crawler.processAll()
        pyclib.forceQuit()
    except:
        traceback.print_exc()
